// Spider - a web-robot class
//
// Copyright (C) 1996 by Jef Poskanzer <jef@acme.com>.  All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
// OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
// SUCH DAMAGE.
//
// Visit the ACME Labs Java page for up-to-date versions of this and other
// fine Java utilities: http://www.acme.com/java/

//
// revision:
// Michael Liao customized for Internet Ads tracking system for IMS.
//

package com.ims.iaa;

import java.util.*;
import java.net.*;
import java.io.*;
import com.ims.iaa.URLUtil;

import Acme.*;


/** A web-robot class.
  * <P>
  * This is an Enumeration class that traverses the web starting at
  * a given URL.  It fetches HTML files and parses them for new
  * URLs to look at.  All files it encounters, HTML or otherwise,
  * are returned by the nextElement() method as a URLConnection.
  * <P>
  * The traversal is breadth-first, and by default it is limited to
  * files at or below the starting point - same protocol, hostname, and
  * initial directory.
  * <P>
  * Because of the security restrictions on applets, this is currently
  * only useful from applications.
  * <P>
  * Sample code:
  * <BLOCKQUOTE><PRE>
  * Enumeration spider = new Acme.Spider( "http:  *some.site.com/whatever/" );
  * while ( spider.hasMoreElements() )
  *     {
  *     URLConnection conn = (URLConnection) spider.nextElement();
  *     // Then do whatever you like with conn:
  *     URL thisUrl = conn.getURL();
  *     String thisUrlStr = thisUrl.toExternalForm();
  *     String mimeType = conn.getContentType();
  *     long changed = conn.getLastModified();
  *     InputStream s = conn.getInputStream();
  *     // Etc. etc. etc., your code here.
  *     }
  * </PRE></BLOCKQUOTE>
  * There are also a couple of methods you can override via a subclass, to
  * control things like the search limits and what gets done with broken links.
  * <P>
  * Sample applications that use Acme.Spider:
  * <UL>
  * <LI> <A HREF="WebList.html">WebList</A> - make a list of the files in a web subtree
  * <LI> <A HREF="WebCopy.html">WebCopy</A> - copy a remote web subtree to the local disk
  * <LI> <A HREF="WebGrep.html">WebGrep</A> - grep a web subtree for a pattern
  * </UL>
  * <P>
  * <A HREF="/resources/classes/Acme/Spider.java">Fetch the software.</A><BR>
  * <A HREF="/resources/classes/Acme.tar.gz">Fetch the entire Acme package.</A>
  * <P>
  * @see Acme.HtmlScanner
  * @see Acme.NoRobots
  */

public class Spider implements Acme.HtmlObserver, Enumeration
{

    // Used only in the default error-reporting routines.  If you override
    // them, this is not used at all.
    /*private*/ protected PrintStream err;
    //delete by ShengMengBai at 2005-04-29
	//private String userAgent;
	
	//Add by ShengMeng Bai at 2004-07-27
	//Used to generate a random number.
	Random m_objIntervalMills = new Random();
	
	//++ add by ShengMengBai at 2005-04-29
	UserAgentAux m_agent = null;
	public void setUserAgentObject(UserAgentAux agent)  {
	    m_agent = agent;
	}
	RobotsTXTAnalyzer m_robotsTxt = null;
	public void setRobotsTXTAnalyzer(RobotsTXTAnalyzer robotsTxt)  {
	    m_robotsTxt = robotsTxt;
	}
	//-- add by ShengMengBai at 2005-04-29
	/*delete by ShengMengBai at 2005-04-29
	public void setUserAgent(String agent)  {
		userAgent = agent;
	}
	*/

    // The spider's state.
    /*private*/ Acme.Queue todo = new Acme.Queue();
    private int todoLimit = 0;
	int totalNumber = 0;
	
    /*private*/ Hashtable done;

    // Temporary state used only between when hasMoreElements and
    // nextElement are called.  Multi-threaded callers should be careful.
    // This seems to be an inherent problem with using Enumeration in
    // a multi-threaded context.
    private SpiderItem item;
    private URL thisUrl;

    // The optional authorization cookie.
    private String auth_cookie = null;

    // The list of Acme.HtmlObservers to add to the HtmlScanner.
    private Vector observers = new Vector();

    // The Robot Exclusion checker.
    private Acme.NoRobots noRobots = new Acme.NoRobots( "Acme.Spider" );
	
	// The maxDepth, default -1 means no limit, added by mliao
	private int maxDepth = -1;


    /// Constructor with no size limits.
    // @param err the error stream
	public Spider( PrintStream err )
	{
		this.err = err;
		done = new Hashtable();
	}

    // added by mliao
	public Spider( PrintStream err, int depth )
    {
    	this( err );
		this.maxDepth = depth;
    }

    /// Constructor with no size limits, and the default error stream.
	public Spider()
	{
		this( System.err );
	}

    /// Constructor with a single URL and no size limits.
    // @param urlStr the URL to start off the enumeration
    // @param err the error stream
    public Spider( String urlStr, PrintStream err ) throws MalformedURLException
	{
		this( err );
		addUrl( urlStr );
	}

    // added by mliao
	public Spider( String urlStr, PrintStream err, int depth ) throws MalformedURLException
    {
    	this( err, depth );
    	addUrl( urlStr );
    }

    /// Constructor with a single URL and no size limits, and the default
    // error stream.
    // @param urlStr the URL to start off the enumeration
    public Spider( String urlStr ) throws MalformedURLException
	{
		this( urlStr, System.err );
	}

    // add by mliao
	public Spider( String urlStr, int depth ) throws MalformedURLException
    {
    	this( urlStr, System.err, depth );
    }
    /// Constructor with size limits.
    // This version lets you specify limits on the todo queue and the
    // done hash-table.  If you are using Spider for a large, multi-site
    // traversal, then you may need to set these limits to avoid running
    // out of memory.  Note that setting a todoLimit means the traversal
    // will not be complete - you may skip some URLs.  And setting the
    // doneLimit means it may re-visit some pages.
    // <P>
    // Guesses at good values for an unlimited traversal: 200000 and 20000.
    // You want the doneLimit pretty small because the hash-table gets checked
    // for every URL, so it will be mostly in memory; the todo queue, on the
    // other hand, is only accessed at the front and back, and so will be
    // mostly paged out.
    // @param urlStr the URL to start off the enumeration
    // @param todoLimit maximum number of URLs to queue for examination
    // @param doneLimit maximum number of URLs to remember having done already
	// @param err the error stream
	public Spider( int todoLimit, int doneLimit, PrintStream err )
	{
		this.err = err;
		this.todoLimit = todoLimit;
		if 	( doneLimit == 0 )
	    	done = new Hashtable();
		else
	    	done = new Acme.LruHashtable( doneLimit );
	}

    /// Constructor with size limits.
    // @param urlStr the URL to start off the enumeration
    // @param todoLimit maximum number of URLs to queue for examination
    // @param doneLimit maximum number of URLs to remember having done already
    public Spider( int todoLimit, int doneLimit )
	{
		this( todoLimit, doneLimit, System.err );
	}


    /// Add a URL to the to-do list.
    public synchronized void addUrl( String urlStr ) throws MalformedURLException
	{
		//if(IaaDebug.Debug) IaaUtil.log( "Spider has collected 1 " + urlStr);
		URL url = URLUtil.getURL(urlStr);
		//Acme.Utils.plainUrl( urlStr );
		String thisUrlStr = url.toExternalForm();
		// the base url strategy change: by mliao, 09/18
		// just use the host name has the baseUrl
		String baseUrlStr = URLUtil.getURL( url, "/" ).toExternalForm();
		/*
		if(IaaDebug.Debug) IaaUtil.log( "Spider has collected 2 " + thisUrlStr);
		// mliao: work around the difference result of 
		// URL.toExternalFrom() between JDK 1.1.8 and JDK1.3
		// (didn't test on any other JDK version );
		if ( thisUrlStr.endsWith( url.getHost() ) ) thisUrlStr = thisUrlStr +"/";
		if(IaaDebug.Debug) IaaUtil.log( "Spider has collected 3 " + thisUrlStr);		
		
		String baseUrlStr = Acme.Utils.baseUrlStr( thisUrlStr );
		if(IaaDebug.Debug) IaaUtil.log( "Spider has collected 4" + baseUrlStr);
		*/		
		if(IaaDebug.ProDebug)totalNumber ++;
		todo.addBack( new SpiderItem( thisUrlStr, null, 0, baseUrlStr ) );
	}

    /// Set the authorization cookie.
    // <P>
    // Syntax is userid:password.
    public synchronized void setAuth( String auth_cookie )
	{
		this.auth_cookie = auth_cookie;
	}

    /// Add an extra observer to the scanners we make.  Multiple observers
    // get called in the order they were added.
    // <P>
    // Alternatively, if you want to add a different observer to each
    // scanner, you can cast the input stream to a scanner and call
    // its add routine, like so:
    // <BLOCKQUOTE><CODE><PRE>
    // InputStream s = conn.getInputStream();
    // Acme.HtmlScanner scanner = (Acme.HtmlScanner) s;
    // scanner.addObserver( this );
    // </PRE></CODE></BLOCKQUOTE>
    public synchronized void addObserver( Acme.HtmlObserver observer )
	{
		observers.addElement( observer );
	}


    /// This method can be overridden by a subclass if you want to change
    // the search policy.  The default version only does URLs that start
    // with the same string as the base URL.  An alternate version might
    // instead go by the search depth.
    protected boolean doThisUrl( String thisUrlStr, int depth, String baseUrlStr )
	{
    //++Add by ShengMeng Bai at 2004-11-09
    //reason: Some site maybe redirect at first page, but the URL is different.
    //		  In this instance, we must crawl this page.
    	if(depth == 0){
    		return true;
    	}
    //--Add by ShengMeng Bai at 2004-11-09
		if ( thisUrlStr.startsWith( baseUrlStr ) )  {
			if ((maxDepth==-1) || (depth< maxDepth ))  // added by mliao
				return true;
		}
		return false;
	}

    /// This method can be overridden by a subclass if you want to change
    // the broken link policy.  The default version reports the broken
    // link on the error stream.  An alternate version might attempt to
    // send mail to the owner of the page with the broken link.
    protected void brokenLink( String fromUrlStr, String toUrlStr, String errmsg )
	{
		//err.println( "Broken link in " + fromUrlStr );
		//err.println( "    pointing to " +toUrlStr );
		//err.println( "    " + errmsg );
	}

    /// This method can be overridden by a subclass if you want to change
    // the error reporting policy.  The default version reports the error
    // link on the error stream.  An alternate version might ignore the error.
    protected void reportError( String fromUrlStr, String toUrlStr, String errmsg )
	{
		err.println( "Error in " + fromUrlStr );
		err.println( "    pointing to " +toUrlStr );
		err.println( "    " + errmsg );
	}

    
    private boolean gotOne = false;

    // Get the next file, if possible.
    private synchronized void getOne()
	{
		while ( ! todo.isEmpty() )
	    {
		    item = (SpiderItem) todo.getFront();

		    // Check again if we've already done this one.
		    if ( ! done.containsKey( item.thisUrlStr ) )
			{
				// commented by mliao -- 08/24
				//done.put( item.thisUrlStr, item.thisUrlStr );
				try
			    {
				    thisUrl = URLUtil.getURL( item.thisUrlStr );
				    //14/Dec/2000 temp comment the following line to disable robots check
				    //if ( ! noRobots.ok( thisUrl ) )	continue;
				    gotOne = true;
					//added by mliao
					/*
					if ( item.fromUrl!=null)  {
						IaaUtil.log("found a link: " + thisUrl + " in " + item.fromUrl );
					} else  {
						IaaUtil.log("found a link: " + thisUrl );
					}
					*/
					//+++
				    return;
			    }
				catch ( MalformedURLException e )
			    {
				    String msg = e.getMessage();
				    if ( checkMalformedURL( msg ) )
					brokenLink( "getOne() MalformedException: "+
					    myUrlToString( item.fromUrl ), item.thisUrlStr,
					    msg );
			    }
				catch ( Exception e )
			    {
				    reportError(
					myUrlToString( item.fromUrl ), item.thisUrlStr,
					e.toString() );
			    }
			}
	    }
		gotOne = false;
		return;
	}
    
    private static boolean checkMalformedURL( String msg )
	{
		// Java is missing protocol handlers for many common
		// protocols.  Ignore those errors.
		if ( msg.startsWith( "unknown protocol: " ) )
		{
		    String protocol = msg.substring( msg.lastIndexOf( ' ' ) + 1 );
		    if ( protocol.equalsIgnoreCase( "gopher" ) ||
		         protocol.equalsIgnoreCase( "ftp" ) ||
		         protocol.equalsIgnoreCase( "file" ) ||
		         protocol.equalsIgnoreCase( "telnet" ) ||
		         protocol.equalsIgnoreCase( "news" ) ||
		         protocol.equalsIgnoreCase( "mailto" ) ||
		         protocol.equalsIgnoreCase( "javascript" ) )
			return false;
		}
		return true;
	}

    /// Standard Enumeration method.
    public synchronized boolean hasMoreElements()
	{
		if ( ! gotOne )
		    getOne();
		return gotOne;
	}


    /// Standard Enumeration method.
    //25 Oct 2000 Anthony Zhang
    // Add this field to record the last connect time of the spider
    private Date connectTime = new Date(0);
    /// Standard Enumeration method.
    public synchronized Object nextElement()
    {
    	if ( ! gotOne )
    	    getOne();
    	if ( ! gotOne )
    	    return null;
    	gotOne = false;

    	// Make local copies of the temporary global state variables, so the
    	// window for overwriting them is smaller.
    	SpiderItem localItem = item;
    	URL localThisUrl = thisUrl;
    	
    	//++ add by ShengMengBai at 2005-04-30
    	if(m_robotsTxt != null && !m_robotsTxt.isAllow(localThisUrl)){
            IaaUtil.log("Obey robots.txt: " + localThisUrl);
    	    return null;
    	}
    	//-- add by ShengMengBai at 2005-04-30

    	try
        {
    	    URLConnection uc = localThisUrl.openConnection();
    	    if ( auth_cookie != null )
    			uc.setRequestProperty( "Authorization", "Basic " + Acme.Utils.base64Encode( auth_cookie ) );
    	    //++change by ShengMengBai at 2005-04-29
    	    String sAgent = m_agent.getUserAgent();
   	        IaaDebug.say(1, "Connect to " + uc.getURL().toString() + "\r\n\t by user agent: " + sAgent);
    		uc.setRequestProperty("User-Agent", sAgent);
//++changsheng for persisent connection
    		uc.setRequestProperty("Connection",   "Keep-Alive");
//--
    		//uc.setRequestProperty("User-Agent", userAgent);
    	    //--change by ShengMengBai at 2005-04-29
    		//25 Oct 2000 Anthony Zhang
    		/** Caculate the interval between now and the last connect time */
    		long intervalMills = System.currentTimeMillis() - connectTime.getTime();
    		//25 Oct 2000 Anthony Zhang
    		/** Compare the interval with the pre set property PageConnectInterval */
//++**********Modify by ShengMeng Bai at 2004-07-27
//    		intervalMills = IaaDebug.PageConnectInterval - intervalMills;
    		int iMaxRandom = IaaDebug.iMaxPageConnectInterval - IaaDebug.iMinPageConnectInterval;
    		if (iMaxRandom > 0){
    			int iTemp = m_objIntervalMills.nextInt(iMaxRandom);
        		intervalMills = iTemp//m_objIntervalMills.nextInt(iMaxRandom)
					+ IaaDebug.iMinPageConnectInterval - intervalMills;
//        		IaaUtil.log( "!!!!!!!!!!!!!!!!!!!!!!Interval value is:" + intervalMills );
//        		IaaUtil.log( "!!!!!!!!!!!!!!!!!!!!!!Random value is:" + iTemp );
    		} else {
        		intervalMills = IaaDebug.iMinPageConnectInterval - intervalMills;
    		}
//--**********Modify by ShengMeng Bai at 2004-07-27
    		//25 Oct 2000 Anthony Zhang
    		/** if the interval is not enough then let the spider sleep for a while */
    		if (intervalMills > 0)Thread.sleep(intervalMills);
    	    connectTime = new Date(System.currentTimeMillis());
    	    IaaDebug.say("Start connect " + uc.getURL());
    	    uc.connect();
    	    IaaDebug.say("End connect " + uc.getURL());
    	    InputStream s = uc.getInputStream();

    		// 08/24 note:
    		// the following revision is to avoid revisiting a redirected url
    		String redirectUrlStr = uc.getURL().toExternalForm(); //added by mliao  - 08/24   	
    		// 09/06 note:
    	    // the following revision is to avoid visiting a redirected url out of scope
    		if ( doThisUrl( redirectUrlStr, item.depth , item.baseUrlStr ) ) //added by mliao - 09/06
    		{ 
    		    if ( ! done.containsKey( redirectUrlStr ) )  { // added by mliao - 08/24
    				// store this redirect url
    				done.put( redirectUrlStr, redirectUrlStr ); //added by mliao - 08/24
    				// and it's original url
    				if ( !redirectUrlStr.equals( localItem.thisUrlStr ) )  {
    					done.put( localItem.thisUrlStr, localItem.thisUrlStr ); //added by mliao - 08/24
    				}

    				String contentType = uc.getContentType();
    			    if ( contentType != null && contentType.indexOf("text/html")!=-1 )
    				{
    					//"if-else" added by liao
    					if ( maxDepth ==-1 | localItem.depth < maxDepth-1 ) {
    						
    						// Make a scanner, and pass in the SpiderItem as the clientData.
    						HtmlScanner scanner = new HtmlScanner(
    						    s, localThisUrl, this, localItem );

    						// Add any extra observers.
    						Enumeration en = observers.elements();
    						while ( en.hasMoreElements() )
    					    {
    						    Acme.HtmlObserver observer =
    							(Acme.HtmlObserver) en.nextElement();
    						    scanner.addObserver( observer );
    					    }

    						// And make a URLConnection that uses this scanner.
    						SpiderConnection asc = new SpiderConnection( uc, scanner, localItem );
    						//++change by ShengMengBai at 2005-04-28
    						//cause: we need logging current depth.
    						//return asc;
    						return new SpiderConnectionGetter(asc, localItem);
    						//++change by ShengMengBai at 2005-04-28
    						
    					} else  {
    						// And make a URLConnection.
    						SpiderConnection asc = new SpiderConnection( uc, s, localItem );
    						//++change by ShengMengBai at 2005-04-28
    						//cause: we need logging current depth.
    						//return asc;
    						return new SpiderConnectionGetter(asc, localItem);
    						//++change by ShengMengBai at 2005-04-28
    					}
    				}
    			    else
    				{
    					// If it's not HTML we don't have to interpose our parser.
						//++change by ShengMengBai at 2005-04-28
						//cause: we need logging current depth.
						//return asc;
						return new SpiderConnectionGetter(uc, localItem);
						//++change by ShengMengBai at 2005-04-28
    				}
    			} else  { //added by mliao - 08/24
    				//already visited, don't visit it again
    				s.close(); //added by mliao - 08/24
    			} //added by mliao - 08/24 
    		} else  { //added by mliao - 09/06
    			//this redirected url is out of scope, don't visit it
    			s.close(); //added by mliao - 09/06
    		} //added by mliao - 09/06
        }
    	catch ( FileNotFoundException e )
        {
    	    brokenLink("nextElement() file not found exception: " +
    		myUrlToString( localItem.fromUrl ), localItem.thisUrlStr,
    		e.getMessage() );
        }
    	catch ( UnknownHostException e )
        {
    	    brokenLink("nextElement() unknown host: " +
    		myUrlToString( localItem.fromUrl ), localItem.thisUrlStr,
    		"unknown host -- " + e.getMessage() );
        }
    	catch ( Exception e )
        {
    	    reportError(
    		myUrlToString( localItem.fromUrl ), localItem.thisUrlStr,
    		e.toString() );
        }
    	return null;
    }
    
    
    private String myUrlToString( URL url )
	{
		if ( url == null )
		    return "an initial URL";
		else
		    return url.toExternalForm();
	}


    /// Acme.HtmlObserver callback.
    public void gotAHREF( String urlStr, URL contextUrl, Object clientData )
	{
		//IaaUtil.log("find a link!");
		add( urlStr, contextUrl, (SpiderItem) clientData );
	}

    /// Acme.HtmlObserver callback.
    public void gotIMGSRC( String urlStr, URL contextUrl, Object clientData )
	{
		//we don't need IMG, by mliao
		//add( urlStr, contextUrl, (SpiderItem) clientData );
	}

    /// Acme.HtmlObserver callback.
    public void gotFRAMESRC( String urlStr, URL contextUrl, Object clientData )
	{
		add( urlStr, contextUrl, (SpiderItem) clientData );
	}

    /// Acme.HtmlObserver callback.
    public void gotBASEHREF( String urlStr, URL contextUrl, Object clientData )
	{
		// Nothing.
	}

    /// Acme.HtmlObserver callback.
    public void gotAREAHREF( String urlStr, URL contextUrl, Object clientData )
	{
		add( urlStr, contextUrl, (SpiderItem) clientData );
	}

    /// Acme.HtmlObserver callback.
    public void gotLINKHREF( String urlStr, URL contextUrl, Object clientData )
	{
		//we don't need Link image, by mliao
		//add( urlStr, contextUrl, (SpiderItem) clientData );
	}

    /// Acme.HtmlObserver callback.
    public void gotBODYBACKGROUND( String urlStr, URL contextUrl, Object clientData )
	{
		//we don't need BACLKGROUP image, by mliao
		//add( urlStr, contextUrl, (SpiderItem) clientData );
	}

    private void add( String urlStr, URL contextUrl, SpiderItem item )
	{
		try
	    {
		    
			/*
			// Convert to no-ref, canonical form.
			URL url = Acme.Utils.plainUrl( contextUrl, urlStr );
			*/
			//replace above line with folloing line by mliao
			URL url = URLUtil.getURL ( contextUrl, urlStr );
			
		    //added by mliao - 09/06
			// 09/06 note: this revision is to remove reference in url
			// the Acme.Utils.plainUrl() does this, but also does more things 
			// that I don't need!
			String fileStr = url.getFile();
			url = new URL( url.getProtocol(), url.getHost(), url.getPort(), fileStr );
			// end of adding - 09/16
			
			urlStr = url.toExternalForm();
		    // Add it.
		    addOne( urlStr, contextUrl, item );
			
			// commented by mliao
			/*
		    // Also add all parent directories up to the root.  We'll get
		    // a lot of duplicates this way, but the hashtable will filter
		    // them out.
		    String rootUrlStr =
			( URLUtil.getURL( URLUtil.getURL( urlStr ), "/" ) ).toExternalForm();
		    while ( urlStr.length() > rootUrlStr.length() )
			{
				int lastSlash = urlStr.lastIndexOf( '/', urlStr.length() - 2 );
				urlStr = urlStr.substring( 0, lastSlash + 1 );
				addOne( urlStr, contextUrl, item );
			} 
			*/ //end of comment -- mliao 
	    }
		catch ( MalformedURLException e )
	    {
		    String msg = e.getMessage();
		    if ( checkMalformedURL( msg ) )
				brokenLink( "add() : " +
					myUrlToString( contextUrl ), urlStr, msg );
	    }
	}
    
    private void addOne( String urlStr, URL contextUrl, SpiderItem item )
	{
		// Check if we've already done this one.
		if ( ! done.containsKey( urlStr ) )
	    {
		    //IaaUtil.log( "scaner found one: " + urlStr + "  context:"+item.thisUrlStr);
		    // Check if we should do this one.
		    if ( doThisUrl( urlStr, item.depth + 1, item.baseUrlStr ) )
			{
				// Yes.
				//IaaUtil.log( "\tadd it" );
		        
		        //++commented by changsheng  2006-7-19 
			/*	if ( todoLimit == 0 || todo.size() < todoLimit )
				    todo.addBack( new SpiderItem(
					urlStr, contextUrl, item.depth + 1,
					item.baseUrlStr ) );
					if(IaaDebug.ProDebug)totalNumber ++;		*/
		        if ( todoLimit == 0 || todo.size() < todoLimit )
		        {
		            SpiderItem $1 = new SpiderItem(
							urlStr, contextUrl, item.depth + 1,
							item.baseUrlStr ) ; 
		            if(todo.contains($1) == false){
					    todo.addBack( $1 );
						if(IaaDebug.ProDebug)totalNumber ++;
		            }
		        }
		        //--
		        
			}
	    }
	}


    /// Test program.  .
    public static void main( String[] args )
	{	
        //System.getProperties().put("proxySet", "true");
		//System.getProperties().put("proxyHost", "yangdan");
		//System.getProperties().put("proxyPort", "3128");
		if ( args.length < 1 )
		{
		    System.err.println( "usage: Spider URL" );
		    return;
		}
		Spider as;
		try
		{
		    as = new Spider( args[0], System.err , 3 );
		    as.setUserAgentObject(UserAgentAux.getInstance("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)"));
		    //as.setUserAgent("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)");
		}
		catch ( MalformedURLException e )
		{
		    System.err.println( e );
		    return;
		}
		
		PrintWriter out = null;

		try  {
			out = new PrintWriter (new FileOutputStream( "spider_result.html" ));
			out.println("<html>");
			out.println("<title>Spider Results</title>");
			out.println("<body>");
			out.println("<table align='center' width=100% border=2 cellpadding=2 cellspacing=0>");
			out.println("<tr><th>Visited URL</th><th>Depth</th><th>Context URL</th></tr>");
			
		} catch (IOException ex ){
			System.err.println( "create file failed" );
		}
		int i = 0 ;
		while ( as.hasMoreElements() )
	    {
		    i++;
		    SpiderConnectionGetter getter = (SpiderConnectionGetter)as.nextElement();
		    if ( getter == null) {
				IaaDebug.say("???One of the getter is null???");
				continue;
			}
		    URLConnection uc = getter.getURLConnection();
			//added by mliao 
			if ( uc == null)  {
				continue;
			}
			
		    URL thisUrl = uc.getURL();
		    String thisUrlStr = thisUrl.toExternalForm();
		    String mimeType = uc.getContentType();

		    int bytes = 0;
			if ( mimeType.indexOf("text/html")!= -1)  {
				
				//record the result;
//				if (uc instanceof SpiderConnection)  {
					SpiderItem si = getter.spiderItem;
					StringWriter writer = new StringWriter();
					//output this url
					writer.write("<tr><td width=50%><a href=\"" + si.thisUrlStr + "\">"+ si.thisUrlStr + "</a>");
					if ( !si.thisUrlStr.equals( thisUrlStr ) )  {
						writer.write("<br>redirect to<br><a href=\""+ thisUrlStr + "\">"+ thisUrlStr + "</a>");
					}
					writer.write("</td>");
					//output this depth of this url
					writer.write("<td>"+ si.depth+"</td>");
					//output the context url
					writer.write("<td>");
					if ( si.fromUrl != null )  
						writer.write("<a href=\""+ si.fromUrl+ "\">"+ si.fromUrl + "</a>");
					else 
						writer.write("entry point");
					writer.write("</td><tr>");
					out.println( writer.toString() );
					out.flush(); 
					
					IaaUtil.log("\tvisit " + thisUrlStr + "  context: " 
							+ si.fromUrl);
//				}
				
				try
				{
					InputStream s = uc.getInputStream();
					//byte[] buf = new byte[4096];
					//int len;
					//while ( ( len = s.read( buf ) ) != -1 )  {
					//	IaaDebug.out.write( buf, 0, len );
					//	bytes += len;
					//}
					s.close();
				} catch ( IOException e ) {
					IaaUtil.log("main() Exception:" + e );
				}
				
			}
	    }
		System.out.println(i);
		out.println("</table>");
		out.println("</body></html>");
		out.close();
		System.exit(0);
	}
}
/**
 * @author ShengMengBai
 *
 * Because we must log current impression depth, 
 * but the return object of Spider.nextElement() may be URLConnection or SpiderConnection,
 * if return URLConnection object, we can't get the SpiderItem object, 
 * so use this class to save the SpiderItem imformation and URLConnection information
 * and return it from Spider.nextElement()
 */
class SpiderConnectionGetter{
    private URLConnection m_uc;
    private int m_iDepth = -1;
    protected SpiderItem spiderItem = null;
    public SpiderConnectionGetter(URLConnection uc, SpiderItem item){
        m_uc = uc;
        spiderItem = item;
        m_iDepth = item.depth;
    }
    public URLConnection getURLConnection(){
        return m_uc;
    }
    public int getDepth(){
        return m_iDepth;
    }
}

// A struct class to hold info on a queued URL.
class SpiderItem
{
    String thisUrlStr;
    URL fromUrl;
    int depth;
    String baseUrlStr;

    public SpiderItem(
	String thisUrlStr, URL fromUrl, int depth, String baseUrlStr )
	{
		this.thisUrlStr = thisUrlStr;
		this.fromUrl = fromUrl;
		this.depth = depth;
		this.baseUrlStr = baseUrlStr;
	}
}


// SpiderConnection - utility class for Spider
//
// A SpiderConnection is the type returned by Spider.  It's a
// URLConnection, slightly modified internally to work with Spider.

class SpiderConnection extends URLConnection
{
    private URLConnection uc;
    protected InputStream s;
	SpiderItem spiderItem; //add by mliao

    public SpiderConnection( URLConnection uc ,SpiderItem item /*add by mliao */) 
		throws IOException
	{
		super( uc.getURL() );
		this.uc = uc;
		this.s = uc.getInputStream();
		this.spiderItem = item;	//add by mliao
	}

    public SpiderConnection( URLConnection uc, InputStream s, SpiderItem item /*add by mliao */)
	{
		super( uc.getURL() );
		this.uc = uc;
		this.s = s;
		this.spiderItem = item; //add by mliao
	}

    public InputStream getInputStream() throws IOException
	{
		return s;
	}

    // The rest just forward to uc's methods.
    final public void connect() throws IOException
	{
		IaaDebug.say("Start connect " + uc.getURL());
		uc.connect();
		IaaDebug.say("End connect " + uc.getURL());
	}
    final public URL getURL()
	{
		return uc.getURL();
	}
    final public int getContentLength()
	{
		return uc.getContentLength();
	}
    final public String getContentType()
	{
		return uc.getContentType();
	}
    final public String getContentEncoding()
	{
		return uc.getContentEncoding();
	}
    final public long getExpiration()
	{
		return uc.getExpiration();
	}
    final public long getDate()
	{
		return uc.getDate();
	}
    final public long getLastModified()
	{
		return uc.getLastModified();
	}
    final public String getHeaderField( String name )
	{
		return uc.getHeaderField( name );
	}
    final public int getHeaderFieldInt( String name, int Default )
	{
		return uc.getHeaderFieldInt( name, Default );
	}
    final public long getHeaderFieldDate( String name, long Default )
	{
		return uc.getHeaderFieldDate( name, Default );
	}
    final public String getHeaderFieldKey( int n )
	{
		return uc.getHeaderFieldKey( n );
	}
    final public String getHeaderField( int n )
	{
		return uc.getHeaderField( n );
	}
    final public Object getContent() throws IOException
	{
		return uc.getContent();
	}
    final public OutputStream getOutputStream() throws IOException
	{
		return uc.getOutputStream();
	}
    final public String toString()
	{
		return uc.toString();
	}
    final public void setDoInput( boolean doinput )
	{
		uc.setDoInput( doinput );
	}
    final public boolean getDoInput()
	{
		return uc.getDoInput();
	}
    final public void setDoOutput( boolean dooutput )
	{
		uc.setDoOutput( dooutput );
	}
    final public boolean getDoOutput()
	{
		return uc.getDoOutput();
	}
    final public void setAllowUserInteraction( boolean allowuserinteraction )
	{
		uc.setAllowUserInteraction( allowuserinteraction );
	}
    final public boolean getAllowUserInteraction()
	{
		return uc.getAllowUserInteraction();
	}
    final public static void setDefaultAllowUserInteraction(
	boolean defaultallowuserinteraction )
	{
		URLConnection.setDefaultAllowUserInteraction(
		    defaultallowuserinteraction );
	}
    final public static boolean getDefaultAllowUserInteraction()
	{
		return URLConnection.getDefaultAllowUserInteraction();
	}
    final public void setUseCaches( boolean usecaches )
	{
		uc.setUseCaches( usecaches );
	}
    final public boolean getUseCaches()
	{
		return uc.getUseCaches();
	}
    final public void setIfModifiedSince( long ifmodifiedsince )
	{
		uc.setIfModifiedSince( ifmodifiedsince );
	}
    final public long getIfModifiedSince()
	{
		return uc.getIfModifiedSince();
	}
    final public boolean getDefaultUseCaches()
	{
		return uc.getDefaultUseCaches();
	}
    final public void setDefaultUseCaches( boolean defaultusecaches )
	{
		uc.setDefaultUseCaches( defaultusecaches );
	}
    final public void setRequestProperty( String key, String value )
	{
		uc.setRequestProperty( key, value );
	}
    final public String getRequestProperty( String key )
	{
		return uc.getRequestProperty( key );
	}
    final public static void setDefaultRequestProperty(
	String key, String value )
	{
		URLConnection.setDefaultRequestProperty( key, value );
	}
    final public static String getDefaultRequestProperty( String key )
	{
		return URLConnection.getDefaultRequestProperty( key );
	}
    
	final public static void setContentHandlerFactory(
	ContentHandlerFactory fac )
	{
		URLConnection.setContentHandlerFactory( fac );
	}
}

